{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import datetime\n",
    "import re\n",
    "import csv\n",
    "import urllib.request\n",
    "import urllib.parse\n",
    "import urllib.robotparser\n",
    "from bs4 import BeautifulSoup\n",
    "import lxml.html\n",
    "from lxml.cssselect import CSSSelector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download(url,user_agent=\"wswp\",proxy=None,num_retries=2):\n",
    "    print('Downloading:',url)\n",
    "    headers = {'User-agent':user_agent}\n",
    "    request = urllib.request.Request(url,headers=headers)\n",
    "    opener = urllib.request.build_opener()\n",
    "    if proxy:\n",
    "        proxy_params = {urllib.parse.urlparse(url).scheme: proxy}\n",
    "        opener.add_handler(urllib.request.ProxyHandler(proxy_params))\n",
    "    try:\n",
    "        html = opener.open(request)\n",
    "    except urllib.request.HTTPError as e:\n",
    "        print('Download error:',e.reason)\n",
    "        html = None\n",
    "        if num_retries>0 and hasattr(e,'code') and 500<=e.code<600:\n",
    "            html = download(url,user_agent,proxy,num_retries-1)\n",
    "    return html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://example.webscraping.com/places/default/view/United-Kingdom-239\n",
      "['<label class=\"readonly\" for=\"places_area\" id=\"places_area__label\">Area: </label>']\n"
     ]
    }
   ],
   "source": [
    "def reTest(url):\n",
    "    html = download(url)\n",
    "    text = html.read().decode('utf-8')\n",
    "    # re.findall('<td class=\"w2p_fw\">(.*?)</td>',html.read().decode('utf-8'))[1]\n",
    "    print(re.findall('<tr id=\"places_area__row\">.*?<td\\s*class=[\"\\']w2p_fl[\"\\']>(.*?)</td>',text))\n",
    "reTest('http://example.webscraping.com/places/default/view/United-Kingdom-239')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://example.webscraping.com/places/default/view/United-Kingdom-239\n",
      "tr: <tr id=\"places_area__row\"><td class=\"w2p_fl\"><label class=\"readonly\" for=\"places_area\" id=\"places_area__label\">Area: </label></td><td class=\"w2p_fw\">244,820 square kilometres</td><td class=\"w2p_fc\"></td></tr>\n",
      "td: <td class=\"w2p_fw\">244,820 square kilometres</td>\n",
      "244,820 square kilometres\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\xuzong\\project\\pycharm\\virtualenv\\default\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file C:\\Program Files\\Python36\\Lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "def bs4Demo():\n",
    "    broken_html = '<ul class=country><li>Area<li>Population</ul>'\n",
    "    # parse the HTML\n",
    "    soup = BeautifulSoup(broken_html,'html.parser')\n",
    "    fixed_html = soup.prettify()\n",
    "    print(fixed_html)\n",
    "    ul = soup.find('ul',attrs={'class':'country'})\n",
    "    print(ul.find('li')) # returns just the first match\n",
    "    print(ul.findAll('li')) # returns all match\n",
    "def bs4Test(url):\n",
    "    html = download(url)\n",
    "    text = html.read().decode('utf-8')\n",
    "    soup = BeautifulSoup(text)\n",
    "    # locate the area row\n",
    "    tr = soup.find(attrs={'id':'places_area__row'})\n",
    "    print('tr:',tr)\n",
    "    td = tr.find(attrs={'class':'w2p_fw'}) # locate the area tag\n",
    "    print('td:',td)\n",
    "    area = td.text # extract the text from this tag\n",
    "    print(area)\n",
    "bs4Test('http://example.webscraping.com/places/default/view/United-Kingdom-239')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<ul class=\"country\">\n",
      "<li>Area</li>\n",
      "<li>Population</li>\n",
      "</ul>\n",
      "\n",
      "AreaPopulation\n",
      "Downloading: http://example.webscraping.com/places/default/view/United-Kingdom-239\n",
      "244,820 square kilometres\n"
     ]
    }
   ],
   "source": [
    "def lxmlDemo():\n",
    "    broken_html = '<ul class=country><li>Area<li>Population</ul>'\n",
    "    # parse the HTML\n",
    "    tree = lxml.html.fromstring(broken_html)\n",
    "    fixed_html = lxml.html.tostring(tree,pretty_print=True)\n",
    "    print(fixed_html.decode('utf-8'))\n",
    "    td = tree.cssselect('ul.country')[0]\n",
    "    print(td.text_content())\n",
    "lxmlDemo()\n",
    "def lxmlTest(url):\n",
    "    html = download(url)\n",
    "    text = html.read().decode('utf-8')\n",
    "    tree = lxml.html.fromstring(text)\n",
    "    td = tree.cssselect('tr#places_area__row > td.w2p_fw')[0]\n",
    "    print(td.text_content())\n",
    "lxmlTest('http://example.webscraping.com/places/default/view/United-Kingdom-239')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "FIELDS = ('area', 'population', 'iso', 'country', 'capital', 'continent', 'tld', 'currency_code', 'currency_name', 'phone', 'postal_code_format', 'postal_code_regex', 'languages', 'neighbours')\n",
    "def re_scraper(html):\n",
    "    results = {}\n",
    "    for field in FIELDS:\n",
    "        results[field] = re.search('<tr id=\"places_%s__row\">.*?<td class=\"w2p_fw\">(.*?)</td>' % field, html).groups()[0]\n",
    "    return results\n",
    "def bs_scraper(html):\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    results = {}\n",
    "    for field in FIELDS:\n",
    "        results[field] = soup.find('table').find('tr',id='places_%s__row' % field).find('td',class_='w2p_fw').text\n",
    "    return results\n",
    "def lxml_scraper(html):\n",
    "    tree = lxml.html.fromstring(html)\n",
    "    results = {}\n",
    "    for field in FIELDS:\n",
    "        results[field] = tree.cssselect('table > tr#places_%s__row > td.w2p_fw' % field)[0].text_content()\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://example.webscraping.com/places/default/view/United-Kingdom-239\n",
      "Regular expressions: 1.62 seconds\n",
      "BeautifulSoup: 12.98 seconds\n",
      "Lxml: 2.54 seconds\n"
     ]
    }
   ],
   "source": [
    "NUM_ITERATIONS = 1000 # number of times to test each scraper\n",
    "html = download('http://example.webscraping.com/places/default/view/United-Kingdom-239').read().decode('utf-8')\n",
    "for name, scraper in [('Regular expressions', re_scraper), ('BeautifulSoup', bs_scraper), ('Lxml', lxml_scraper)]:\n",
    "    start = time.time() # record start time of scrape\n",
    "    for i in range(NUM_ITERATIONS):\n",
    "        if scraper == re_scraper:\n",
    "            re.purge()\n",
    "        result = scraper(html)\n",
    "        # check scraped result is as expected\n",
    "        assert(result['area'] == '244,820 square kilometres')\n",
    "    end = time.time()\n",
    "    print('%s: %.2f seconds' % (name, end-start)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Throttle:\n",
    "    \"\"\"\n",
    "    Add a delay between downloads to the same domain\n",
    "    \"\"\"\n",
    "    def __init__(self,delay):\n",
    "        # amount of delay between downloads for each domain\n",
    "        self.delay = delay\n",
    "        # timestamp of when a domain was last accessed\n",
    "        self.domains = {}\n",
    "        \n",
    "    def wait(self,url):\n",
    "        domain = urllib.parse.urlparse(url).netloc\n",
    "        last_accessed = self.domains.get(domain)\n",
    "        if self.delay > 0 and last_accessed is not None:\n",
    "            sleep_secs = self.delay - (datetime.datetime.now() - last_accessed).seconds\n",
    "            if sleep_secs > 0:\n",
    "                # domain has been accessed recently so need to sleep\n",
    "                time.sleep(sleep_secs)\n",
    "        # update the last accessed time\n",
    "        self.domains[domain] = datetime.datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CsrapeCallback:\n",
    "    def __init__(self):\n",
    "        self.f = open('countries.csv', 'w')\n",
    "        self.writer = csv.writer(self.f)\n",
    "        self.fields = ('area', 'population', 'iso', 'country', 'capital', 'continent', 'tld', 'currency_code', 'currency_name', 'phone', 'postal_code_format', 'postal_code_regex', 'languages', 'neighbours')\n",
    "        self.writer.writerow(self.fields)\n",
    "    def __call__(self, url, html):\n",
    "        if re.search('/view/', url):\n",
    "            tree = lxml.html.fromstring(html)\n",
    "            row = []\n",
    "            for fields in self.fields:\n",
    "                row.append(tree.cssselect('table > tr#places_{}__row > td.w2p_fw'.format(fields))[0].text_content())\n",
    "            self.writer.writerow(row)\n",
    "    def __del__(self):\n",
    "        self.f.close()\n",
    "        print('结束')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://example.webscraping.com\n",
      "Downloading: http://example.webscraping.com/places/default/index/1\n",
      "Downloading: http://example.webscraping.com/places/default/view/Antigua-and-Barbuda-10\n",
      "http://example.webscraping.com/places/default/view/Antigua-and-Barbuda-10 ['443 square kilometres', '86,754', 'AG', 'Antigua and Barbuda', \"St. John's\", 'NA', '.ag', 'XCD', 'Dollar', '+1-268', '', '', 'en-AG', ' ']\n",
      "Downloading: http://example.webscraping.com/places/default/view/Antarctica-9\n",
      "http://example.webscraping.com/places/default/view/Antarctica-9 ['14,000,000 square kilometres', '0', 'AQ', 'Antarctica', '', 'AN', '.aq', '', '', '', '', '', '', ' ']\n",
      "Downloading: http://example.webscraping.com/places/default/view/Anguilla-8\n",
      "http://example.webscraping.com/places/default/view/Anguilla-8 ['102 square kilometres', '13,254', 'AI', 'Anguilla', 'The Valley', 'NA', '.ai', 'XCD', 'Dollar', '+1-264', '', '', 'en-AI', ' ']\n",
      "Downloading: http://example.webscraping.com/places/default/view/Angola-7\n",
      "http://example.webscraping.com/places/default/view/Angola-7 ['1,246,700 square kilometres', '13,068,161', 'AO', 'Angola', 'Luanda', 'AF', '.ao', 'AOA', 'Kwanza', '244', '', '', 'pt-AO', 'CD NA ZM CG ']\n",
      "Downloading: http://example.webscraping.com/places/default/view/Andorra-6\n",
      "http://example.webscraping.com/places/default/view/Andorra-6 ['468 square kilometres', '84,000', 'AD', 'Andorra', 'Andorra la Vella', 'EU', '.ad', 'EUR', 'Euro', '376', 'AD###', '^(?:AD)*(\\\\d{3})$', 'ca', 'ES FR ']\n",
      "Downloading: http://example.webscraping.com/places/default/view/American-Samoa-5\n",
      "http://example.webscraping.com/places/default/view/American-Samoa-5 ['199 square kilometres', '57,881', 'AS', 'American Samoa', 'Pago Pago', 'OC', '.as', 'USD', 'Dollar', '+1-684', '', '', 'en-AS,sm,to', ' ']\n",
      "Downloading: http://example.webscraping.com/places/default/view/Algeria-4\n",
      "http://example.webscraping.com/places/default/view/Algeria-4 ['2,381,740 square kilometres', '34,586,184', 'DZ', 'Algeria', 'Algiers', 'AF', '.dz', 'DZD', 'Dinar', '213', '#####', '^(\\\\d{5})$', 'ar-DZ', 'NE EH LY MR TN MA ML ']\n",
      "Downloading: http://example.webscraping.com/places/default/view/Albania-3\n",
      "http://example.webscraping.com/places/default/view/Albania-3 ['28,748 square kilometres', '2,986,952', 'AL', 'Albania', 'Tirana', 'EU', '.al', 'ALL', 'Lek', '355', '', '', 'sq,el', 'MK GR CS ME RS XK ']\n",
      "Downloading: http://example.webscraping.com/places/default/view/Aland-Islands-2\n",
      "http://example.webscraping.com/places/default/view/Aland-Islands-2 ['1,580 square kilometres', '26,711', 'AX', 'Aland Islands', 'Mariehamn', 'EU', '.ax', 'EUR', 'Euro', '+358-18', '#####', '^(?:FI)*(\\\\d{5})$', 'sv-AX', ' ']\n",
      "Downloading: http://example.webscraping.com/places/default/view/Afghanistan-1\n",
      "http://example.webscraping.com/places/default/view/Afghanistan-1 ['647,500 square kilometres', '29,121,286', 'AF', 'Afghanistan', 'Kabul', 'AS', '.af', 'AFN', 'Afghani', '93', '', '', 'fa-AF,ps,uz-AF,tk', 'TM CN IR TJ PK UZ ']\n",
      "Downloading: http://example.webscraping.com/places/default/index\n"
     ]
    }
   ],
   "source": [
    "def link_crawler(send_url,link_regex,delay=1,max_depth=2,scrape_callback=None):\n",
    "    \"\"\"\n",
    "    Crawl from the given seed URL following links matched by link_regex\n",
    "    \"\"\"\n",
    "    throttle = Throttle(delay)\n",
    "    rp = urllib.robotparser.RobotFileParser()\n",
    "    rp.set_url(urllib.parse.urljoin(send_url,'/robots.txt'))\n",
    "    rp.read()\n",
    "    user_agent = \"Baiduspider\"\n",
    "    crawl_queue = [send_url]\n",
    "    seen = {send_url:0}\n",
    "    while crawl_queue:\n",
    "        url = crawl_queue.pop()\n",
    "        depth = seen[url]\n",
    "        if depth != max_depth:\n",
    "            # chec url passes robots.txt restrictions\n",
    "            if rp.can_fetch(user_agent,url):\n",
    "                throttle.wait(url)\n",
    "                html = download(url,user_agent,None,2).read().decode('utf-8')\n",
    "                if html == None:\n",
    "                    continue;\n",
    "                links = []\n",
    "                if scrape_callback:\n",
    "                    scrape_callback(url, html) or []\n",
    "                if link_regex:\n",
    "                    # filter for links matching our regular expression\n",
    "                    links.extend(link for link in get_links(html) if re.match(link_regex, link))\n",
    "                    # print('肯定会执行')\n",
    "                for link in links:\n",
    "                    # from absolute link\n",
    "                    link = urllib.parse.urljoin(send_url,link)\n",
    "                    # check if have alreday seen this link\n",
    "                    if link not in seen:\n",
    "                        seen[link] = depth + 1\n",
    "                        crawl_queue.append(link)\n",
    "            else:\n",
    "                print('Blocked by robots.txt:',url)\n",
    "def get_links(html):\n",
    "    \"\"\"\n",
    "    Return a list of links from html\n",
    "    \"\"\"\n",
    "    # a regular expression to extract all links from the webpage\n",
    "    webpage_regex = re.compile('<a[^>]+href=[\"\\'](.*?)[\"\\']',re.IGNORECASE)\n",
    "    # list of all links from the webpaeg\n",
    "    return webpage_regex.findall(html)\n",
    "def scrape_callback(url, html):\n",
    "    if re.search('/view/', url):\n",
    "        tree = lxml.html.fromstring(html)\n",
    "        row = [tree.cssselect('table > tr#places_%s__row > td.w2p_fw' % field)[0].text_content() for field in FIELDS]\n",
    "        print(url, row)\n",
    "link_crawler('http://example.webscraping.com','/places/default/(view|index)',scrape_callback=scrape_callback)\n",
    "# link_crawler('http://httpstat.us/500','/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "list1=[1,2,3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "list1.extend([4,5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4, 5]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
